#!/bin/bash

Deploy_Path=`cd $(dirname $0); pwd -P`

#####################################################################
#source $Deploy_Path/lib/init_ceph_lib.sh
#从第一个mon节点scp ceph.conf文件到当前节点
get_ceph_conf(){
    for ((i=0; 1<5; i++));do
        scp -P $ssh_port  $Deploy_Node:/etc/ceph/$Cluster_Name.conf /etc/ceph/$Cluster_Name.conf && chown -R ceph:ceph /etc/ceph
        if [ "$?"x == "0"x ];then
            echo "第一个mon节点配置文件已生成！"
            break
        else
            echo "第一个mon节点配置文件未生成！等待10s后自动重试......"
            sleep 10s
        fi
    done
}

#从已存在的mon服务运行的节点scp client.admin.keyring文件到当前节点，并将属主和数组改为ceph用户ceph组（在扩容的mon节点执行）
get_client_admin_keyring() {
    if [ -f /etc/ceph/$Cluster_Name.client.admin.keyring ];then
        echo "The file /etc/ceph/$Cluster_Name.client.admin.keyring already exists."
    else
        scp -P $ssh_port $Deploy_Node:/etc/ceph/$Cluster_Name.client.admin.keyring /etc/ceph/$Cluster_Name.client.admin.keyring && chown -R ceph:ceph /etc/ceph
    fi
}

#从已存在的mon服务运行的节点scp client.bootstrap-osd keyring文件到当前节点，并将属主和数组改为ceph用户ceph组（在扩容的mon节点执行）
get_bootstrap_osd_keyring() {
    #ceph --cluster $Cluster_Name auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring && chown -R ceph:ceph /var/lib/ceph  #弃用，当ceph-mon master节点没起来之前改命令无法获取keyring，徒增等待时间
    if [ -f /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring ];then
        echo "The file /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring already exists."
    else
        scp -P $ssh_port $Deploy_Node:/var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring && chown -R ceph:ceph /var/lib/ceph
        cp -f /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring /etc/ceph/ && chown -R ceph:ceph /etc/ceph
    fi
}

systemctl_daemon_reload(){
    systemctl daemon-reload
}

ceph_crash_achive_all(){
    if ! ceph crash archive-all;then
        echo "ceph crash archive-all is faild!"
    fi
}

######################################################################
#start init mon service
#

#将所有ceph工作目录都设置为ceph用户ceph组
chown_ceph_pwd() {
    chown -R ceph:ceph /var/lib/ceph
    chown -R ceph:ceph /etc/ceph
}

#检查并创建ceph工作目录
check_mon_workpath(){
    #在节点上设置/var/lib/ceph目录及其子文件属主为ceph用户ceph组
    if [ ! -d /etc/ceph ];then
        mkdir -p /etc/ceph
    fi
    if [ ! -d /var/lib/ceph/mon/$Cluster_Name-$MON_NAME -o ! -d /var/lib/ceph/bootstrap-mon ];then
        mkdir -p /var/lib/ceph/mon/$Cluster_Name-$MON_NAME
        mkdir -p /var/lib/ceph/bootstrap-mon
        MON_Workpath_Check=no
    else
        MON_Workpath_Check=yes
    fi
    if [ ! -d /var/lib/ceph/bootstrap-osd ];then
        mkdir -p /var/lib/ceph/bootstrap-osd
    fi
    if [ ! -d /var/lib/ceph/bootstrap-mgr ];then
        mkdir -p /var/lib/ceph/bootstrap-mgr
    fi
    if [ ! -d /var/lib/ceph/bootstrap-mds ];then
        mkdir -p /var/lib/ceph/bootstrap-mds
    fi
    if [ ! -d /var/lib/ceph/bootstrap-rgw ];then
        mkdir -p /var/lib/ceph/bootstrap-rgw
    fi
    if [ ! -d /var/lib/ceph/bootstrap-rbd ];then
        mkdir -p /var/lib/ceph/bootstrap-rbd
    fi
    if [ ! -d /var/lib/ceph/bootstrap-rbd-mirror ];then
        mkdir -p /var/lib/ceph/bootstrap-rbd-mirror
    fi
    if [ ! -d /var/lib/ceph/mgr ];then
        mkdir -p /var/lib/ceph/mgr
    fi
    if [ ! -d /var/lib/ceph/mds ];then
        mkdir -p /var/lib/ceph/mds
    fi
    chown_ceph_pwd
}

#仅在第一个mon节点执行
ceph_env_check() {
    #在所有ceph节点上设置/etc/ceph目录及其子文件属主为ceph用户ceph组
    if [ -f /etc/ceph/ceph-cluster.env ];then
        source /etc/ceph/ceph-cluster.env
        echo "ceph environment file check done"
    else
        check_mon_workpath
        if [ "$Deploy_Mode"x == binaryx ];then
            if [ -f $Deploy_Path/conf/ceph-cluster.env ];then
                cp -f $Deploy_Path/conf/ceph-cluster.env /etc/ceph/ceph-cluster.env  && chown -R ceph:ceph /etc/ceph
                source /etc/ceph/ceph-cluster.env
            else
                echo "Error: The ceph environment file $Deploy_Path/conf/ceph-cluster.env is not found!"
                exit 1
            fi
        else
            echo "Error: The ceph environment file $Deploy_Path/conf/ceph-cluster.env is not found!"
            exit 1
        fi
    fi
}

#仅在第一个mon节点执行
ceph_conf_check() {
    #在所有ceph节点上设置/etc/ceph目录及其子文件属主为ceph用户ceph组
    if [ -f /etc/ceph/$Cluster_Name.conf ];then
        echo "ceph configure file check done"
    else
        if [ -f $Deploy_Path/conf/$Cluster_Name.conf ];then
            cp -f $Deploy_Path/conf/$Cluster_Name.conf /etc/ceph/$Cluster_Name.conf  && chown -R ceph:ceph /etc/ceph
        else
            echo "Error: The ceph configure file $Deploy_Path/conf/ceph.conf is not found!"
            exit 1
        fi
    fi
}

#create ceph.conf
modify_ceph_conf() {
    #修改文件中mon_host列表
    MON_HOSTS=$(echo $MON_Nodes|sed 's/[ ]/,/g')
    sed -i 's/^mon_host = .*/mon_host = '"$MON_HOSTS"'/g' /etc/ceph/$Cluster_Name.conf
    echo /etc/ceph/$Cluster_Name.conf
    #修改配置文件中cluster_network的值
    sed -i 's#^cluster_network.*#cluster_network = '"$Cluster_CIDR"'#g' /etc/ceph/$Cluster_Name.conf
    echo /etc/ceph/$Cluster_Name.conf
    #修改配置文件中public_network的值
    sed -i 's#^public_network.*#public_network = '"$Public_CIDR"'#g' /etc/ceph/$Cluster_Name.conf
    #生成ceph集群fsid,并在ceph.conf配置文件中修改集群fsid为当前生成的fsid
    FSID=`uuidgen`
    sed -i 's/^fsid.*/fsid = '"$FSID"'/g' /etc/ceph/$Cluster_Name.conf
    sed -i 's/^auth_client_required.*/auth_client_required = '"$Auth_Mode"'/g' /etc/ceph/$Cluster_Name.conf
    sed -i 's/^auth_cluster_required.*/auth_cluster_required = '"$Auth_Mode"'/g' /etc/ceph/$Cluster_Name.conf
    sed -i 's/^auth_service_required.*/auth_service_required = '"$Auth_Mode"'/g' /etc/ceph/$Cluster_Name.conf
    echo /etc/ceph/$Cluster_Name.conf
}

#使用第一个mon的名称、ip地址、端口等信息创建一个monmap文件（仅在第一个mon节点执行）
create_monmap() {
    #创建集群monmap
    if [ -f /var/lib/ceph/bootstrap-mon/$Cluster_Name-monmap ];then
        monmaptool --create --add $MON_NAME $MON_IP --fsid $FSID /var/lib/ceph/bootstrap-mon/$Cluster_Name-monmap
    fi
}

#从已存在的mon服务中获取现行monmap并输出到文件（在扩容的mon节点执行）

#使用当前扩容mon节点的mon名称、ip地址、端口等信息向已有的monmap文件中添加mon信息（在扩容的mon节点执行）
#add_monmap() {
#    monmaptool --add $MON_NAME $MON_IP  /var/lib/ceph/tmp/$Cluster_Name-monmap
#}

#(函数迁移至init_ceph_lib.sh公共库中)
#从已存在的mon服务运行的节点scp client.admin.keyring文件到当前节点，并将属主和数组改为ceph用户ceph组（在扩容的mon节点执行）
#get_client_admin_keyring() {
#scp $Deploy_Node:/etc/ceph/$Cluster_Name.client.admin.keyring /etc/ceph/$Cluster_Name.client.admin.keyring && chown -R ceph:ceph /etc/ceph
#}

#从已存在的mon服务运行的节点scp mon.keyring文件到当前节点，并将属主和数组改为ceph用户ceph组（在扩容的mon节点执行）
get_mon_keyring() {
    if [ -f /var/lib/ceph/tmp/$Cluster_Name.mon.keyring ];then
        scp -P $ssh_port $Deploy_Node:/var/lib/ceph/tmp/$Cluster_Name.mon.keyring /var/lib/ceph/tmp/$Cluster_Name.mon.keyring && chown -R ceph:ceph /var/lib/ceph
    fi
    #ceph --cluster $Cluster_Name auth get mon. -o /var/lib/ceph/tmp/$Cluster_Name.mon.keyring && chown -R ceph:ceph /var/lib/ceph
}

#(函数迁移至init_ceph_lib.sh公共库中)
#从已存在的mon服务运行的节点scp client.bootstrap-osd keyring文件到当前节点，并将属主和数组改为ceph用户ceph组（在扩容的mon节点执行）
#get_bootstrap_osd_keyring() {
    #ceph --cluster $Cluster_Name auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring && chown -R ceph:ceph /var/lib/ceph
#scp $Deploy_Node:/var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring && chown -R ceph:ceph /var/lib/ceph
#}

check_mon_id(){
    if ceph --connect-timeout 1 mon dump |egrep -w "^[0-9]+: \[.*\] mon.$MON_NAME$";then
        MON_ID_Ready=yes
    else
        MON_ID_Ready=no
    fi
}

check_port_listen(){
    if ss -tnlp |grep -w ':6789';then
        port_status=used
    else
        port_status=available
    fi
}

#新版本不需要额外执行ceph mon add命令，该函数没有被使用
add_mon() {
    ceph --cluster $Cluster_Name mon add $MON_NAME $MON_IP
}

get_monmap(){
    if [ -f /var/lib/ceph/bootstrap-mon/$Cluster_Name-monmap ];then
        ceph --cluster $Cluster_Name mon getmap -o  /var/lib/ceph/bootstrap-mon/$Cluster_Name-monmap
    fi
}

#始化mon数据库
init_mon() {
    #此步骤为真正初始化mon服务的步骤,使用 ceph-mon --mkfs 命令初始化一个mon节点，由下面命令行的参数可知 -i后面是要创建的mon节点名称，--monmap表示将指定的monmap导入到新创建的mon节点的数据库中，--keyring表示将指定的keyring导入到新创建的mon节点的数据库中
    sudo -u ceph ceph-mon --cluster $Cluster_Name --mkfs -i $MON_NAME --monmap /var/lib/ceph/tmp/$Cluster_Name-monmap --keyring /var/lib/ceph/tmp/$Cluster_Name.mon.keyring && sleep 5s || exit 1
}

#修改mon的systemd unit文件中的集群名称，适用于自定义ceph集群名称的部署
modify_mon_systemd_unit() {
    deb_name=`ls -l /var/cache/apt/archives/|egrep ceph-mon*.deb`
    #获取mon service unit文件绝对路径
    #MON_Service_File=`rpm -ql ceph-mon|grep -w 'ceph-mon@.service'`
    MON_Service_File=`dpkg -c $deb_name|grep -w 'ceph-mon@.service'`
    #获取mon service unit文件所在目录
    Service_Path=`dirname $MON_Service_File`
    #获取mon service unit文件中默认配置的集群名称
    Source_Cluster=`sed -n 's/Environment=CLUSTER=//p' $MON_Service_File`
    #按当前部署ceph的集群名称克隆并修改mon的systemd unit文件，默认为ceph时不做修改
    if [ "$Source_Cluster"x == "$Cluster_Name"x ];then
        echo "当前ceph cluster name无需修改。"
    else
        cp -a -f $MON_Service_File $Service_Path/$Cluster_Name-ceph-mon@.service
        sed -i "s/Environment=CLUSTER=.*/Environment=CLUSTER=$Cluster_Name/g" $Service_Path/$Cluster_Name-ceph-mon@.service
    fi
}

#创建放行mon服务的防火墙规则
allow_mon_firewall() {
    firewall-cmd --zone=public --add-service=ceph-mon
    firewall-cmd --zone=public --add-service=ceph-mon --permanent
}

#启动mon服务并将mon服务设置为自启动服务
enable_mon_service() {
    if [ "$Deploy_Mode"x == binaryx ];then
        if [ "$Cluster_Name"x == "ceph"x ];then
            systemctl daemon-reload && systemctl start ceph-mon@$MON_NAME && systemctl enable ceph-mon@$MON_NAME
        else
            systemctl daemon-reload && systemctl start $Cluster_Name-ceph-mon@$MON_NAME && systemctl enable $Cluster_Name-ceph-mon@$MON_NAME
        fi
    elif [ "$Deploy_Mode"x == dockerx ];then
        ceph-mon -f --cluster ceph --id $MON_NAME --setuser ceph --setgroup ceph
    elif [ "$Deploy_Mode"x == k8sx ];then
        ceph-mon -f --cluster ceph --id $MON_NAME --setuser ceph --setgroup ceph
        echo "check done"
    else
        echo "Error: The ceph mon init workflow error!"
        exit 1
    fi
}

#为mon通信启用msg2协议
enable_mon_msgr2(){
    if [ "$Enable_Mgsr2"x == yesx ];then
        ceph --cluster $Cluster_Name mon enable-msgr2
    else
        #ceph health mute MON_MSGR2_NOT_ENABLED --sticky
        echo "The msgr2 is disabled."
    fi
}

#禁用允许不安全的mon global_id reclaim通信
disable_auth_allow_insecure_global_id_reclaim(){
    if [ "$Disable_Insecure_Global_Id_Reclaim"x == yesx ];then
        ceph --cluster $Cluster_Name config set mon auth_allow_insecure_global_id_reclaim false
        echo "The auth_allow_insecure_global_id_reclaim is mute."
        #ceph --cluster $Cluster_Name health mute AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED --sticky
    fi
}

stop_mon(){
    if [ "$Deploy_Mode"x == binaryx ];then
        if [ "$Cluster_Name"x == "ceph"x ];then
            systemctl stop ceph-mon@$MON_NAME && systemctl disable ceph-mon@$MON_NAME
        else
            systemctl daemon-reload && systemctl start $Cluster_Name-ceph-mon@$MON_NAME && systemctl enable $Cluster_Name-ceph-mon@$MON_NAME
        fi
    elif [ "$Deploy_Mode"x == dockerx ];then
        ceph mon ok-to-stop $MON_NAME
        sleep 5s
        if MON_SVC=`ps -ef|egrep "^ceph .* /usr/bin/ceph-mon -f --cluster ceph --id $MON_NAME --setuser ceph --setgroup ceph$"`;then
            MON_SVC_PID=`echo $MON_SVC|awk '{print $2}'`
            kill -9 $MON_SVC_PID
        fi
    elif [ "$Deploy_Mode"x == k8sx ];then
        ceph mon ok-to-stop $MON_NAME 
        sleep 5s
        if MON_SVC=`ps -ef|egrep "^ceph .* /usr/bin/ceph-mon -f --cluster ceph --id $MON_NAME --setuser ceph --setgroup ceph$"`;then
            MON_SVC_PID=`echo $MON_SVC|awk '{print $2}'`
            kill -9 $MON_SVC_PID
        fi
    else
        echo "Error: The ceph mon init workflow error!"
        exit 1
    fi
}

rm_mon(){
    if ceph mon dump|grep $MON_NAME;then
        ceph mon rm $MON_NAME
    else
        echo "mon $MON_NAME is not found"
    fi
}

rm_mon_auth(){
    if ceph auth get mon.$MON_NAME ;then
        ceph auth rm mon.$MON_NAME
    else
        echo "mon $MON_NAME keyring is not found"
    fi
}

rm_mon_pwd(){
    rm -rf /var/lib/ceph/mon/ceph-$MON_NAME
}

check_ceph_status() {
    while ! ceph --connect-timeout 1 -s do
        echo "ceph cluster is health ok"
    done
}

mon_env_check(){
    ceph_env_check
    ceph_conf_check
    check_mon_workpath
    if echo $MON_Nodes|grep -w "$Node_FQDN";then
        check_mon_id
        if [ "$MON_ID_Ready"x == nox ];then
            if [ "$MON_Workpath_Check"x == nox ];then
                MON_Status=creating
                echo "MON_Status=creating" > /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env
            else
                if [ -f /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env ];then
                    source /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env
                    if [ "$MON_Status"x == creatingx -o "$MON_Status"x == rebuildx -o "$MON_Status"x == destroyx -o "$MON_Status"x == readyx ];then
                        echo "$MON_Status"
                    else
                        cat /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env
                        echo "Error: The mon workflow is unknown"
                        exit 1
                    fi
                else
                    MON_Status=creating
                    echo "MON_Status=creating" > /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env
                fi
            fi
        elif [ "$MON_ID_Ready"x == yesx ];then
            if [ "$MON_Workpath_Check"x == nox ];then
                if [ -f /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env ];then
                    source /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env
                    if [ "$MON_Status"x == rebuildx -o "$MON_Status"x == destroyx ];then
                        echo "$MON_Status"
                    else
                        cat /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env
                        echo "Error: The mon workflow is unknown"
                        exit 1
                    fi
                else
                    echo "Error: The mon workpath is not found"
                    exit 1
                fi
            elif [ "$MON_Workpath_Check"x == yesx ];then
                if [ -f /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env ];then
                    source /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env
                    if [ "$MON_Status"x == rebuildx -o "$MON_Status"x == destroyx -o "$MON_Status"x == readyx ];then
                        echo "$MON_Status"
                    else
                        cat /var/lib/ceph/mon/ceph-$MON_NAME/$MON_NAME.env
                        echo "Error: The mon workflow is unknown"
                        exit 1
                    fi
                else
                    echo "Error: The mon workpath is not found"
                    exit 1
                fi
            else
                echo "Error: The mon workflow is unknown"
                exit 1
            fi
        else 
            echo "Error: Then MON_ID_Ready is Unknown"
            exit 1
        fi
    else
        echo "Error: Then current node is not mon running node"
        exit 1
    fi
                
        
}

mon_status_check(){
    if (ss -tnlp|egrep ":6789 .* users:\(\(\"ceph-mon\"") && (ss -tnlp|egrep ":3300 .* users:\(\(\"ceph-mon\"");then
        MON_SVC_Status=running
    else
        MON_SVC_Status=norunning
    fi
}

mon_create_workflow(){
    modify_ceph_conf
    #判断当前节点是否为第一个mon节点，如果是则执行mon首节点初始化函数，否则执行mon扩容节点函数
    #echo $Node_FQDN|grep -w $Deploy_Node
    if [ "$Node_FQDN"x == "$Deploy_Node"x -a "$MON_Status"x == creatingx ];then
        create_monmap
    fi
    if [ "$Deploy_Mode"x != k8sx ];then 
        get_ceph_conf
        get_client_admin_keyring
        get_mon_keyring
        get_bootstrap_osd_keyring
    fi
    check_ceph_status
    if [ "$Node_FQDN"x != "$Deploy_Node"x ];then
        add_mon
        get_monmap
    fi
    chown_ceph_pwd
    init_mon
    if [ "$OS_Type"x == centosx ];then
        allow_mds_firewall
    elif [ "$OS_Type"x == opencloudosx ];then
        allow_mds_firewall
    else
        echo "ubuntu firewall to pre-init-system-all-nodes.sh disable."
    fi
    #modify_mon_systemd_unit
    enable_mon_service
    check_ceph_status
    if [ "$Node_FQDN"x == "$Deploy_Node"x ];then
        enable_mon_msgr2
        disable_auth_allow_insecure_global_id_reclaim
    fi
    check_ceph_status
    sleep 2s
    ceph -s
}

mon_rebuild_workflow(){
    mkdir_mon_pwd
    if [ "$Deploy_Mode"x != k8sx ];then
        get_ceph_conf
        get_client_admin_keyring
        get_mon_keyring
        get_bootstrap_osd_keyring
    fi
    check_ceph_status
    add_mon
    get_monmap
    chown_ceph_pwd
    init_mon
    if [ "$OS_Type"x == centosx ];then
        allow_mds_firewall
    elif [ "$OS_Type"x == opencloudosx ];then
        allow_mds_firewall
    else
        echo "ubuntu firewall to pre-init-system-all-nodes.sh disable."
    fi
    #modify_mon_systemd_unit
    enable_mon_service
    check_ceph_status
    if [ "$Node_FQDN"x == "$Deploy_Node"x ];then
        enable_mon_msgr2
        disable_auth_allow_insecure_global_id_reclaim
    fi
    check_ceph_status
    sleep 2s
    ceph -s
}

mon_destroy_workflow(){
   stop_mon
   rm_mon
   rm_mon_auth
   rm_mon_pwd
}

mon_start_workflow(){
    if [ "$Deploy_Mode"x == binaryx ];then
        systemctl start ceph-mon@$MON_NAME
    elif [ "$Deploy_Mode"x == dockerx ];then
        ceph-mon -f --cluster ceph --id $MON_NAME --setuser ceph --setgroup ceph
    elif [ "$Deploy_Mode"x == k8sx ];then
        ceph-mon -f --cluster ceph --id $MON_NAME --setuser ceph --setgroup ceph
    else
        echo "Error: The Deploy_Mode is unknown"
        exit 1
    fi
}
init_mon_workflow() {
    mon_status_check
    mon_env_check
    if [ "$MON_SVC_Status"x == runningx ];then
        case $MON_Status in
            creating)
                echo "Error: Then mon service status is running"
                exit 1
                ;;
            rebuild)
                #mon_destroy_workflow
                #mon_create_workflow
                echo "Error: Then mon service status is running"
                exit 1
                ;;
            ready)
                echo "Error: Then mon service status is running"
                exit 1
                ;;
            destroy)
                echo "Error: Then mon service status is running"
                exit 1
                ;;
            *)
                echo "Error: The mon workflow is unknown"
                exit 1
                ;;
        esac
    elif [ "$MON_SVC_Status"x == norunningx ];then
        case $MON_Status in
            creating)
                mon_create_workflow
                ;;
            rebuild)
                mon_rebuild_workflow
                ;;
            ready)
                mon_start_workflow
                ;;
            destroy)
                mon_destroy_workflow
                ;;
            *)
                echo "Error: The mon workflow is unknown"
                exit 1
                ;;
        esac
    else
        echo "Error: Then mon service status is unknown"
        exit 1
    fi
}
#
#init mon service done
#############################################################################
init_mon_workflow
